{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6a7edc9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from onekey_algo.custom.components.comp1 import draw_roc\n",
    "from onekey_algo.custom.components import metrics\n",
    "import matplotlib.pyplot as plt\n",
    "from onekey_algo import get_param_in_cwd\n",
    "\n",
    "metric_results = []\n",
    "label_data = pd.read_csv(r't1_group.csv')\n",
    "save_dir = get_param_in_cwd('save_dir', 'img')\n",
    "prefix = r'C:/Users/onekey/Desktop/onekey_comp/comp8-Modules/data/3classes/Task1'\n",
    "models = ['SVM', 'MLP', 'LightGBM']\n",
    "os.makedirs(save_dir, exist_ok=True)\n",
    "for model in models:\n",
    "    all_pred = []\n",
    "    all_gt = []\n",
    "    all_labels = []\n",
    "    for subset in ['train', 'test']:\n",
    "        val_log = pd.read_csv(f'{prefix}_{model}_{subset}.csv')\n",
    "        val_log = pd.merge(val_log, label_data, on='ID', how='inner')\n",
    "        ul_labels = np.unique(val_log['label'])\n",
    "        for ul in ul_labels:\n",
    "            pred_score = val_log[f\"label-{ul}\"]\n",
    "            gt = [1 if gt_ == ul else 0 for gt_ in np.array(val_log['label'])]\n",
    "            acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres = metrics.analysis_pred_binary(gt, pred_score)\n",
    "            ci = f\"{ci[0]:.4f}-{ci[1]:.4f}\"\n",
    "            metric_results.append([model, acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres, f\"{subset}-label{ul}\"])\n",
    "\n",
    "            all_pred.append(np.array(pred_score))\n",
    "            all_gt.append(gt)\n",
    "            all_labels.append(f\"{subset}-{ul}\")\n",
    "\n",
    "    draw_roc(all_gt, all_pred, labels=all_labels, title=f\"Model: {model}\", ls=['-', '-', '-', ':', ':', ':'])\n",
    "    plt.savefig(os.path.join(save_dir, f'model_{model}_roc.svg'), bbox_inches = 'tight')\n",
    "    plt.show()\n",
    "metrics = pd.DataFrame(metric_results, \n",
    "                       columns=['ModelName', 'Acc', 'AUC', '95% CI', 'Sensitivity', 'Specificity', 'PPV', 'NPV', \n",
    "                                'Precision', 'Recall', 'F1', 'Threshold', 'Cohort'])\n",
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28adf760",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
